package org.example

import com.hankcs.hanlp.HanLP
import com.hankcs.hanlp.dictionary.stopword.CoreStopWordDictionary
import com.hankcs.hanlp.tokenizer.StandardTokenizer
import com.hankcs.hanlp.summary.TextRankKeyword
import org.apache.spark.sql.SparkSession
import org.apache.spark.api.java.JavaRDD.fromRDD

import scala.collection.convert.ImplicitConversions.`collection AsScalaIterable`
import scala.jdk.CollectionConverters.CollectionHasAsScala

object data1_words {
  def main(args: Array[String]): Unit = {
    val spark = SparkSession
          .builder
          .master("local[*]")
          .appName("spark")
          .getOrCreate()
    val sc = spark.sparkContext
    sc.textFile("E:\\hnl\\scala09\\Scala\\src\\main\\resources\\word1.txt")
      .flatMap(_.split(""))
      .map(x => (x, 1))
      .reduceByKey((x,y) => x + y)
      .foreach(println)
 //中文分词
      val chinese1=HanLP.segment("身高一米七八年收入15万人老实话不多")
      println(chinese1)
      println(chinese1.asScala.map(_.word.trim))
 //标准分词
      val chinese2 = StandardTokenizer.segment("清明节放假++五一劳动节放假")
      println(chinese2)
      println(chinese2.asScala.map(_.word.replaceAll("\\s+","")))
 //关键词提取
      TextRankKeyword.getKeywordList("速看！广东这五个地方将要拆迁，征收位置、面积、补偿和安 置标准都已公布，快来看看有你家吗",5)
        .forEach(println)
      val chinese3 = """16:00:00 pm 23510205030109 [spark大数据]"
  www.baidu.com""".split("\\s+")
      println(chinese3(2).replaceAll("\\[|\\]",""))
    val textArr = Array(
    "su7爆燃遇难者母亲清空相关微博",
    "甲亢哥被重庆甜妹教育了",
    "胡歌老婆是他前助理",
    "金价再起飞",
    "你能接受与伴侣的年龄差是多少"
    )
    val textRDD = sc.parallelize(textArr)
    val textResult = textRDD.map{
      text =>
      val keyword = TextRankKeyword.getKeywordList(text,5).toString
      val words = transform(text)
        (text ,keyword, words)
    } // RDD[(String, String, List[String])]
     textResult.take(1).foreach(println)
    }
  def transform(sentense:String):List[String] ={
    val list = StandardTokenizer.segment(sentense)
    CoreStopWordDictionary.apply(list)
    list.map(x => x.word.replaceAll(" ","")).toList
  }
}
